# coding:utf-8
import jieba.posseg
import random
import re

import threading
import time


# 前面的是噪音句 后面的是原句

def delete_random(line):
    pox = ['uj', 'u', 'ud', 'vn', 'ul', 'd', 'ad', 'dg', 'y', 'z', 'uv', 'uz', 'vd']
    lines = jieba.posseg.cut(line)
    list_r = []
    # 对列表循环同时获取索引
    for num, word in enumerate(lines):
        if word.flag in pox:
            # 获取要删除的词性的索引集合
            list_r.append(num)
        # print(num, word, word.flag)
    # 定义最终需要删除的随机索引集合
    list_need_delete = []
    # 随机选择一半的弱词性的删除，返回的是索引
    list_random = random.sample(range(0, len(list_r)), len(list_r) // 2)
    for k in list_random:
        list_need_delete.append(list_r[k])
    # print(list_r, len(list_r))
    # print(list_random, type(list_random))
    out = ""
    # print(list_need_delete, len(list_r))
    newlines = jieba.posseg.cut(line)
    # 否定词不删
    set_not = ["不", "非", "有", "无", "莫"]
    for num2, word2 in enumerate(newlines):
        if num2 not in list_need_delete or word2 in set_not:
            out += word2.word
    return out


def handle(in1, out1):
    fp = open(in1, 'r', encoding='utf-8')
    fout = open(out1, 'a', encoding='utf-8')

    i = 0
    for line in fp.readlines():
        i += 1
        set_l = line.strip().split("\t")
        # 过滤掉英文行
        if bool(re.search("[a-zA-Z]{10,}", set_l[0])):
            continue
        if bool(re.search("[-\d()*+/=\.]{10,}", set_l[0])):
            fout.write('%s' % line)
            print("这行不加噪处理：", i)
            continue
        result = delete_random(set_l[0])
        fout.write('%s\n' % (result + "\t" + set_l[1]))


if __name__ == '__main__':
    # for i in range(20):
    #     infile = './test0' + str(i) + '.tsv'
    #     outfile = './out.tsv'
    #     t = threading.Thread(target=handle(infile, outfile), args=(5,))
    #     t.start()
    infile = 'g:/target2.txt'
    outfile = 'g:/out.tsv'
    handle(infile, outfile)
